<?php
/**
 * @author qingyu8@staff.weibo.com
 * @date 2018/01/12
 */

namespace console\controllers;

use PHPUnit\Framework\Exception;
use yii;
use yii\console\Controller;
use GuzzleHttp;
use MongoDB;
use GuzzleHttp\Cookie;
use Symfony\Component\DomCrawler\Crawler;
use yii\helpers\Json;

set_time_limit(0);

class InnfinsController extends Controller {

    public $lang = 'zh';
    public $continueLast = 0;
    const COOKIE = 'Aq3TXo_think_lang7=zh; Aq3TXo_think_coin2=CNY; PHPSESSID=kst3ek2htt6no1jspas6luqbq5; Aq3TXo_history1=%5B%22195%22%5D; INNFINS_VISIT_FIRST=1; intercom-id-i3ya9ul7=8a2b507f-57e0-4b8d-adb4-8b73d2b93983; Hm_lvt_d22524b0609c87f1ef27b1c374eaf793=1517107913,1517794796,1518317304,1518331828; Hm_lpvt_d22524b0609c87f1ef27b1c374eaf793=1518331828; _pk_id.1.ad69=7b41abcd5d57d797.1518330033.2.1518334569.1518334569.; _pk_ses.1.ad69=*';
    public $baseHeader = [
        'Accept' => '*/*',
        'Accept-Language' => 'zh',
        'Accept-Encoding' => 'gzip, deflate',
        'User-Agent' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36',
        'Cookie' => self::COOKIE,
        'Host' => 'innfins.com',
        'Connection' => 'keep-alive',
    ];
    /**
     * @var GuzzleHttp\Client
     */
    public $guzzleClient;

    public $mongo;
    public $collection;
    const IDS = [
        0 => '195',
        1 => '196',
        2 => '200',
        3 => '201',
        4 => '204',
        5 => '209',
        6 => '212',
        7 => '213',
        8 => '210',
        9 => '45',
        10 => '51',
        11 => '46',
        12 => '203',
        13 => '63',
        14 => '175',
        15 => '177',
        16 => '13',
        17 => '15',
        18 => '9',
        19 => '14',
        20 => '2',
        21 => '7',
        22 => '211',
        23 => '186',
        24 => '190',
        25 => '6',
        26 => '188',
        27 => '20',
        28 => '189',
        29 => '161',
        30 => '25',
        31 => '101',
        32 => '73',
        33 => '4',
        34 => '12',
        35 => '18',
        36 => '19',
        37 => '24',
        38 => '26',
        39 => '27',
        40 => '28',
        41 => '30',
        42 => '32',
        43 => '34',
        44 => '39',
        45 => '50',
        46 => '52',
        47 => '53',
        48 => '56',
        49 => '58',
        50 => '60',
        51 => '65',
        52 => '69',
        53 => '70',
        54 => '79',
        55 => '81',
        56 => '83',
        57 => '91',
        58 => '93',
        59 => '95',
        60 => '115',
        61 => '119',
        62 => '123',
        63 => '137',
        64 => '159',
        65 => '187',
        66 => '193',
        67 => '197',
        68 => '198',
        69 => '199',
        70 => '149',
        71 => '155',
        72 => '157',
        73 => '194',
        74 => '153',
        75 => '1',
        76 => '135',
        77 => '57',
        78 => '31',
        79 => '205',
        80 => '145',
        81 => '165',
        82 => '55',
        83 => '169',
        84 => '105',
        85 => '21',
        86 => '192',
        87 => '78',
        88 => '64',
        89 => '173',
        90 => '42',
        91 => '202',
        92 => '29',
        93 => '3',
        94 => '5',
        95 => '43',
        96 => '36',
        97 => '17',
        98 => '22',
        99 => '33',
        100 => '37',
        101 => '38',
        102 => '40',
        103 => '41',
        104 => '44',
        105 => '47',
        106 => '48',
        107 => '49',
        108 => '54',
        109 => '59',
        110 => '61',
        111 => '62',
        112 => '67',
        113 => '68',
        114 => '71',
        115 => '75',
        116 => '77',
        117 => '84',
        118 => '97',
        119 => '99',
        120 => '103',
        121 => '107',
        122 => '121',
        123 => '125',
        124 => '127',
        125 => '129',
        126 => '131',
        127 => '133',
        128 => '139',
        129 => '151',
        130 => '167',
        131 => '171',
        132 => '35',
    ];
    public $configFile;
    public $config;
    const CRAWL_INTERVAL = [
        'avaliable_room' => 300,
    ];

    public function beforeAction($action) {
        $this->baseHeader['Accept-Language'] = $this->lang == 'zh' ? 'zh-CN,zh;q=0.9' : 'en;q=0.8';
        $this->guzzleClient = new GuzzleHttp\Client();
        $this->mongo = new MongoDB\Client("mongodb://39.106.175.227:27017");
        $this->collection = $this->lang == 'zh' ? $this->mongo->dive->innfins : $this->mongo->dive->innfins_en;
        $this->configFile = Yii::$app->basePath . '/runtime/innfins-config.php';
        $this->config = require($this->configFile);
        return parent::beforeAction($action);
    }

    public function options($actionID) {
        $params = [
            'lang',
            'continueLast',
        ];
        return array_merge($params, parent::options($actionID));
    }

    public function optionAliases() {
        return ['l' => 'lang'];
    }

    public function actionIndex() {
        echo date('Y-m-d H:i:s') . "|Lang: " . $this->lang . "\n";
        $collection_shipment = $this->lang == 'zh' ? $this->mongo->dive->innfins_shipment : $this->mongo->dive->innfins_shipment_en;
        $collection_line = $this->lang == 'zh' ? $this->mongo->dive->innfins_line : $this->mongo->dive->innfins_line_en;
        $collection_shipmentroom = $this->lang == 'zh' ? $this->mongo->dive->innfins_shipmentroom : $this->mongo->dive->innfins_shipmentroom_en;

        $url = 'http://innfins.com/main/searchApi/listsV2';
        $page = 1;

        $params = [
            'destination' => 11,
            'shipmentnum' => '6',
            'order' => 0,
            'pageinfo' => 1,
            'pnum' => 15,
            'page' => $page,
        ];

        while (1) {
            echo date('Y-m-d H:i:s') . "|page $page\n";
            usleep(mt_rand(800, 2800));
            $res = $this->guzzleClient->request('GET', $url, [
                'query' => array_merge($params, ['page' => $page]),
                'headers' => array_merge($this->baseHeader, [
                    'Referer' => 'http://innfins.com/zh/search?k=&sdate=',
                    'X-Requested-With' => 'XMLHttpRequest',
                ]),
            ]);
            $body = $res->getBody();
            $body = json_decode($body, true);
            if (empty($body) || empty($body['data'])) {
                echo date('Y-m-d H:i:s') . "|empty data\n";
                return;
            }
            foreach ($body['data'] as $k => $v) {
                $item = $this->collection->findOne(['_id' => $v['ship_id']], ['typeMap' => []]);
                //if (!empty($item)) {continue;}

                $v = $this->formatShipV2($v);

                $old = json_decode(json_encode($item), true);
                if (empty($old)) {
                    echo date('Y-m-d H:i:s') . "|ship {$v['ship_id']} new\n";
                    $result = $this->collection->updateOne(['_id' => $v['ship_id']], ['$set' => $v], ['upsert' => true]);
                } else {
                    ksort($old);
                    ksort($v);
                    if ($old != $v) {
                        echo date('Y-m-d H:i:s') . "|ship {$v['ship_id']} diff\n";
                        $result = $this->collection->updateOne(['_id' => $v['ship_id']], ['$set' => $v], ['upsert' => true]);
                    } else {
                        echo date('Y-m-d H:i:s') . "|ship {$v['ship_id']} same\n";
                    }
                }
            }
            $page++;
        }
    }

    public function actionTest() {
        $this->shipment();
    }

    public function saveConfig() {
        file_put_contents($this->configFile, '<?php return ' . var_export($this->config, true) . ';');
    }

    public function curIndex() {
        $lastIndex = $this->config[$this->lang]['ship_id_last_index'];
        if (is_null($lastIndex)) {
            $curIndex = 0;
        } elseif ($this->continueLast) {
            $curIndex = $lastIndex;
        } else {
            $tmp = self::IDS[++$lastIndex];
            $curIndex = isset($tmp) ? $lastIndex : 0;
        }
        $this->config[$this->lang]['ship_id_last_index'] = $curIndex;
        $this->saveConfig();
        return $curIndex;
    }

    public function curId() {
        return self::IDS[$this->curIndex()];
    }

    public function shipmentroom() {
        $id = $this->curId();
        echo "ship $id avaliable_room start\n";

        $data = $this->collection->findOne(['_id' => $id], [
            'projection' => [
                'shipment' => 1,
                '_id' => 0
            ],
        ]);
        //$data = json_decode(Json::encode($data), true);
        $shipments = $data['shipment'];
        foreach ($shipments as $k => $shipment) {
            if (!empty($shipment['crawl_time']) && time() - $shipment['crawl_time'] < self::CRAWL_INTERVAL['avaliable_room']) {
                continue;
            }
            if (!empty($shipment['deleted'])) {
                continue;
            }
            usleep(mt_rand(800, 2800));
            $tmp = $this->guzzleClient->request('GET', 'http://innfins.com/main/Index/shipmentroom?ship_id=' . $id . '&sdate=' . $shipment['sdate'], [
                'headers' => array_merge($this->baseHeader, [
                    //'Referer' => $ship_url,
                    'X-Requested-With' => 'XMLHttpRequest',
                ]),
                'timeout' => 5,
                //                'proxy' => [
                //                    'http'  => 'tcp://218.10.236.6:3128', // Use this proxy with "http"
                //                    //'https' => 'tcp://localhost:9124', // Use this proxy with "https",
                //                    //'no' => ['.mit.edu', 'foo.com']    // Don't use a proxy with these
                //                ]
            ]);
            $ret1 = json_decode((string)$tmp->getBody(), true);
            if (!isset($ret1['code']) || $ret1['code'] != 1) {
                if ($ret1['info'] == '船期不存在') {
                    $shipments[$k]['deleted'] = 1;
                    $shipments[$k]['crawl_time'] = time();
                    $this->collection->updateOne(['_id' => $id], ['$set' => ['shipment' => $shipments]]);
                    continue;
                } else {
                    echo date('Y-m-d H:i:s') . "|shipment {$shipment['_id']} of {$shipment['sdate']} error:" . var_export($ret1, true) . "\n";
                    return;
                }
            } elseif (empty($ret1['data'])) {
                echo date('Y-m-d H:i:s') . "|empty room for shipment {$shipment['_id']} of {$shipment['sdate']}\n";
                continue;
            }
            $avaliable_room = [];
            foreach ($ret1['data'] as $item1) {
                $avaliable_room[] = [
                    'id' => $item1['id'],
                    'discount' => $item1['discount'],
                    'price' => $item1['price'],
                    //剩余数
                    'num_surplus' => $item1['num_surplus'],
                ];
            }
            $shipments[$k]['avaliable_room'] = $avaliable_room;
            $shipments[$k]['crawl_time'] = time();
            $this->collection->updateOne(['_id' => $id], ['$set' => ['shipment' => $shipments]]);
        }
        echo "ship $id avaliable_room end\n";
    }

    //先更新船期，关键信息是sdate，查询剩余船舱的接口需用
    public function shipment() {
        $id = $this->curId();
        echo "ship $id shipment start\n";
        $data = $this->collection->findOne(['_id' => $id], [
            'projection' => [
                'shipment' => 1,
                'ship_url' => 1,
            ],
        ]);
        $data = json_decode(Json::encode($data), true);
        usleep(mt_rand(800, 2800));
        $tmp = $this->guzzleClient->request('GET', 'http://innfins.com/main/index/allmonth?ship_id=' . $id, [
            'headers' => array_merge($this->baseHeader, [
                'Referer' => $data['ship_url'],
                'X-Requested-With' => 'XMLHttpRequest',
            ])
        ]);
        $ret = json_decode((string)$tmp->getBody(), true);
        if ($ret['code'] != 1) {
            echo date('Y-m-d H:i:s') . "|error months:" . var_export($ret, true) . "\n";
            return;
        }
        foreach ($ret['data'] as $month) {
            usleep(mt_rand(800, 2800));
            $tmp = $this->guzzleClient->request('POST', 'http://innfins.com/main/index/shipment', [
                'form_params' => [
                    'ship_id' => $id,
                    'ym' => $month['value'],
                    'coin' => 'EUR'
                ],
                'headers' => array_merge($this->baseHeader, [
                    'Referer' => $data['ship_url'],
                    'X-Requested-With' => 'XMLHttpRequest',
                ]),
            ]);
            $ret = json_decode((string)$tmp->getBody(), true);
            if ($ret['code'] != 1) {
                echo date('Y-m-d H:i:s') . "|error shipment for {$month['value']}\n";
                return;
            }
            if (empty($ret['data'])) {
                echo date('Y-m-d H:i:s') . "|empty shipment for {$month['value']}\n";
                continue;
            }
            foreach ($ret['data'] as $item) {
                $shipment_new = [
                    'id' => $item['id'],
                    'ship_id' => $item['ship_id'],
                    'name' => $item['lname'],
                    //航线id
                    'line_id' => $item['line_id'],
                    'day' => $item['day'],
                    'length' => $item['length'],
                    'dives' => $item['dives'],
                    'price' => $item['price'],
                    'sdate' => $item['sdate'],
                    'edate' => $item['edate'],
                ];
                $hit = false;
                foreach ($data['shipment'] as $k => $shipment) {
                    if ($shipment['id'] == $shipment_new['id']) {//修改
                        $hit = true;
                        $shipment['crawl_time'] = date('Y-m-d H:i:s');
                        $data['shipment'][$k] = array_merge($shipment, $shipment_new);
                    }
                }
                if (!$hit) {//新增
                    $data['shipment'][] = $shipment_new;
                }
            }
            $this->collection->updateOne(['_id' => $id], ['$set' => ['shipment' => array_values($data['shipment'])]]);
        }
        echo "ship $id shipment end\n";
    }

    public function formatShip($v) {
        echo date('Y-m-d H:i:s') . "|formatShip {$v['ship_id']} Start\n";
        $ship_url = 'http://innfins.com' . $v['url'];
        usleep(mt_rand(800, 2800));
        $r = $this->guzzleClient->request('GET', $ship_url, [
            'headers' => array_merge($this->baseHeader)
        ]);
        $body = $r->getBody();
        $crawler = new \Symfony\Component\DomCrawler\Crawler((string)$body, $ship_url);
        $subCrawler = $crawler->filter('.gallery-list>ul>li>img');
        $imgs = [];
        foreach ($subCrawler as $node) {
            $src = $node->getAttribute('data-ori');
            $pos = strpos($src, '!c');
            $src = $pos ? substr($src, 0, $pos) : $src;
            $imgs[] = $src;
        }
        $basicInfo = [];
        $subCrawler = $crawler->filter('.feature li');
        foreach ($subCrawler as $node) {
            $field = $node->getAttribute('class');
            $basicInfo[$field] = $node->lastChild->textContent;
        }
        $desc_brief = $crawler->filter('.cont-box>.brief')->getNode(0)->textContent;
        $desc_all = $crawler->filter('.cont-box>.all')->getNode(0)->textContent;
        $price = $crawler->filter('.hd>.from>.num')->getNode(0)->textContent;
        usleep(mt_rand(800, 2800));
        $tmp = $this->guzzleClient->request('GET', 'http://innfins.com/main/index/allmonth?ship_id=' . $v['ship_id'], [
            'headers' => array_merge($this->baseHeader, [
                'Referer' => $ship_url,
                'X-Requested-With' => 'XMLHttpRequest',
            ]),
        ]);
        $ret = json_decode((string)$tmp->getBody(), true);
        if ($ret['code'] != 1) {
            throw new Exception("error months:" . var_export($ret, true));
        }
        $shipment = $lines = $shipmentroom = [];//船期
        foreach ($ret['data'] as $month) {
            usleep(mt_rand(800, 2800));
            $tmp = $this->guzzleClient->request('POST', 'http://innfins.com/main/index/shipment', [
                'form_params' => [
                    'ship_id' => $v['ship_id'],
                    'ym' => $month['value'],
                    'coin' => 'EUR'
                ],
                'headers' => array_merge($this->baseHeader, [
                    'Referer' => $ship_url,
                    'X-Requested-With' => 'XMLHttpRequest',
                ]),
            ]);
            $ret = json_decode((string)$tmp->getBody(), true);
            if ($ret['code'] != 1) {
                throw new Exception(date('Y-m-d H:i:s') . "|error shipment for {$month['value']}:" . var_export($ret, true));
            }
            if (empty($ret['data'])) {
                echo date('Y-m-d H:i:s') . "|empty shipment for {$month['value']}\n";
                continue;
            }
            foreach ($ret['data'] as $item) {
                $shipment[] = [
                    '_id' => $item['id'],
                    'ship_id' => $item['ship_id'],
                    'name' => $item['lname'],
                    //航线id
                    'line_id' => $item['line_id'],
                    'day' => $item['day'],
                    'length' => $item['length'],
                    'dives' => $item['dives'],
                    'price' => $item['price'],
                    'sdate' => $item['sdate'],
                    'edate' => $item['edate'],
                ];
                //房间
                usleep(mt_rand(800, 2800));
                $tmp = $this->guzzleClient->request('GET', 'http://innfins.com/main/Index/shipmentroom?ship_id=' . $v['ship_id'] . '&sdate=' . $item['sdate'], [
                    'headers' => array_merge($this->baseHeader, [
                        'Referer' => $ship_url,
                        'X-Requested-With' => 'XMLHttpRequest',
                    ]),
                ]);
                $ret1 = json_decode((string)$tmp->getBody(), true);
                if ($ret1['code'] != 1) {
                    exit(date('Y-m-d H:i:s') . "|error lines for shipment {$item['id']} of {$month['value']}\n");
                }
                if (empty($ret1['data'])) {
                    echo date('Y-m-d H:i:s') . "|empty lines for shipment {$item['id']} of {$month['value']}\n";
                    continue;
                }
                foreach ($ret1['data'] as $item1) {
                    $photos = [];
                    if (!empty($item1['photos'])) {
                        foreach ($item1['photos'] as $photo) {
                            $photos[] = $photo['url'];
                        }
                    }
                    $shipmentroom[] = [
                        'id' => $item1['id'],
                        'ship_id' => $item1['ship_id'],
                        'name' => $item1['name'],
                        //船期id
                        'smid' => $item1['smid'],
                        'content' => $item1['content'],
                        'discount' => $item1['discount'],
                        'photos' => $photos,
                        'scoin' => $item1['scoin'],
                        'price' => $item1['price'],
                        //剩余数
                        'num_surplus' => $item1['num_surplus'],
                        'thumb' => $item1['thumb']
                    ];
                }
            }

            //航线
            usleep(mt_rand(800, 2800));
            $tmp = $this->guzzleClient->request('GET', 'http://innfins.com/main/Index/getlinesforym?ship_id=' . $v['ship_id'] . '&ym=' . $month['value'], [
                'headers' => array_merge($this->baseHeader, [
                    'Referer' => $ship_url,
                    'X-Requested-With' => 'XMLHttpRequest',
                ]),
            ]);
            $ret = json_decode((string)$tmp->getBody(), true);
            if ($ret['state'] != 'success') {
                exit(date('Y-m-d H:i:s') . "|error lines for {$month['value']}\n");
            }
            if (empty($ret['data'])) {
                echo date('Y-m-d H:i:s') . "|empty lines for {$month['value']}\n";
                continue;
            }
            foreach ($ret['data'] as $item) {
                $photos = [];
                if (!empty($item['photos'])) {
                    foreach ($item['photos'] as $photo) {
                        $photos[] = $photo['url'];
                    }
                }
                $lines[] = [
                    'id' => $item['id'],
                    'ship_id' => $item['ship_id'],
                    'name' => $item['name'],
                    'length' => $item['length'],
                    'dives' => $item['dives'],
                    'requires' => $item['requires'],
                    'photos' => $photos,
                    'departure' => $item['departure'],
                    'description' => $item['description'],
                    'return_port' => $item['return_port'],
                ];
            }
        }

        $subCrawler = $crawler->filter('.explanation>.itm>span');
        $explanation = [];
        foreach ($subCrawler as $node) {
            $field = trim($node->textContent);
            $val = trim($node->nextSibling->nextSibling->textContent);
            $explanation[$field] = $val;
        }

        $subCrawler = $crawler->filter('.section-feature>.bd>.itm');
        $feature = [];
        foreach ($subCrawler as $node) {
            $grandcrawler = new \Symfony\Component\DomCrawler\Crawler($node, $ship_url);
            $field = $grandcrawler->filter('.name')->getNode(0)->textContent;
            $vals = [];
            $valNodes = $grandcrawler->filter('.cont>ul>li');
            foreach ($valNodes as $liNode) {
                $vals[] = $liNode->textContent;
            }
            $feature[$field] = $vals;
        }

        $photos = [];
        foreach ($v['photo'] as $photo) {
            $photos[] = $photo['url'];
        }
        $res = [
            '_id' => $v['ship_id'],
            'ship_id' => $v['ship_id'],
            'name' => $v['name'],
            'ename' => $v['ename'],
            'cdestination' => $v['cdestination'],
            'create_y' => $v['create_y'],
            'photo' => $v['photo'],
            'evaluation_sentence' => $v['evaluation_sentence'],
            'imgs' => $imgs,
            'basic_info' => $basicInfo,
            'desc_brief' => $desc_brief,
            'desc_all' => $desc_all,
            'price' => $price,
            'shipment' => $shipment,
            'lines' => $lines,
            'shipmentroom' => $shipmentroom,
            'feature' => $feature,
            'explanation' => $explanation,
        ];
        echo date('Y-m-d H:i:s') . "|formatShip {$v['ship_id']} End\n";
        return $res;
    }

    public function ship() {
        $id = $this->curId();
        echo "ship $id start\n";
        echo "ship $id end\n";
    }

    public function formatShipV2($v) {
        echo date('Y-m-d H:i:s') . "|formatShip {$v['ship_id']} Start\n";
        $ship_url = 'http://innfins.com' . $v['url'];
        usleep(mt_rand(800, 2800));
        $r = $this->guzzleClient->request('GET', $ship_url, [
            'header' => array_merge($this->baseHeader)
        ]);
        $body = $r->getBody();
        $crawler = new \Symfony\Component\DomCrawler\Crawler((string)$body, $ship_url);
        $subCrawler = $crawler->filter('.gallery-list>ul>li>img');
        $imgs = [];
        foreach ($subCrawler as $node) {
            $src = $node->getAttribute('data-ori');
            $pos = strpos($src, '!c');
            $src = $pos ? substr($src, 0, $pos) : $src;
            $imgs[] = $src;
        }
        $basicInfo = [];
        $subCrawler = $crawler->filter('.feature li');
        foreach ($subCrawler as $node) {
            $field = $node->getAttribute('class');
            $basicInfo[$field] = $node->lastChild->textContent;
        }
        $desc_brief = $crawler->filter('.cont-box>.brief')->getNode(0)->textContent;
        $desc_all = $crawler->filter('.cont-box>.all')->getNode(0)->textContent;
        $price = $crawler->filter('.hd>.from>.num')->getNode(0)->textContent;
        usleep(mt_rand(800, 2800));
        $tmp = $this->guzzleClient->request('GET', 'http://innfins.com/main/index/allmonth?ship_id=' . $v['ship_id'], [
            'headers' => array_merge($this->baseHeader, [
                'Referer' => $ship_url,
                'X-Requested-With' => 'XMLHttpRequest',
            ])
        ]);
        $ret = json_decode((string)$tmp->getBody(), true);
        if ($ret['code'] != 1) {
            exit(date('Y-m-d H:i:s') . "|error months:" . var_export($ret, true) . "\n");
        }
        $shipment = $lines = $shipmentroom = [];//船期
        $line_ids = [];
        $shipmentroom_writedown = 0;
        foreach ($ret['data'] as $month) {
            usleep(mt_rand(800, 2800));
            $tmp = $this->guzzleClient->request('POST', 'http://innfins.com/main/index/shipment', [
                'form_params' => [
                    'ship_id' => $v['ship_id'],
                    'ym' => $month['value'],
                    'coin' => 'EUR'
                ],
                'headers' => array_merge($this->baseHeader, [
                    'Referer' => $ship_url,
                    'X-Requested-With' => 'XMLHttpRequest',
                ]),
            ]);
            $ret = json_decode((string)$tmp->getBody(), true);
            if ($ret['code'] != 1) {
                exit(date('Y-m-d H:i:s') . "|error shipment for {$month['value']}\n");
            }
            if (empty($ret['data'])) {
                echo date('Y-m-d H:i:s') . "|empty shipment for {$month['value']}\n";
                continue;
            }
            foreach ($ret['data'] as $item) {
                $avaliable_room = [];
                //房间
                usleep(mt_rand(800, 2800));
                $tmp = $this->guzzleClient->request('GET', 'http://innfins.com/main/Index/shipmentroom?ship_id=' . $v['ship_id'] . '&sdate=' . $item['sdate'], [
                    'headers' => array_merge($this->baseHeader, [
                        'Referer' => $ship_url,
                        'X-Requested-With' => 'XMLHttpRequest',
                    ]),
                ]);
                $ret1 = json_decode((string)$tmp->getBody(), true);
                if ($ret1['code'] != 1) {
                    exit(date('Y-m-d H:i:s') . "|error room for shipment {$item['id']} of {$month['value']}\n");
                }
                if (empty($ret1['data'])) {
                    echo date('Y-m-d H:i:s') . "|empty room for shipment {$item['id']} of {$month['value']}\n";
                    continue;
                }
                foreach ($ret1['data'] as $item1) {
                    $photos = [];
                    if (!empty($item1['photos'])) {
                        foreach ($item1['photos'] as $photo) {
                            $photos[] = $photo['url'];
                        }
                    }
                    if ($shipmentroom_writedown === 0) {
                        $shipmentroom[] = [
                            'id' => $item1['id'],
                            'name' => $item1['name'],
                            'content' => $item1['content'],
                            'photos' => $photos,
                            'scoin' => $item1['scoin'],
                            'thumb' => $item1['thumb']
                        ];
                    }
                    $avaliable_room[] = [
                        'id' => $item1['id'],
                        'discount' => $item1['discount'],
                        'price' => $item1['price'],
                        //剩余数
                        'num_surplus' => $item1['num_surplus'],
                    ];
                }
                if (!empty($shipmentroom) && $shipmentroom_writedown === 0) {
                    $shipmentroom_writedown = 1;
                }

                $shipment[] = [
                    '_id' => $item['id'],
                    'ship_id' => $item['ship_id'],
                    'name' => $item['lname'],
                    //航线id
                    'line_id' => $item['line_id'],
                    'day' => $item['day'],
                    'length' => $item['length'],
                    'dives' => $item['dives'],
                    'price' => $item['price'],
                    'sdate' => $item['sdate'],
                    'edate' => $item['edate'],
                    'avaliable_room' => $avaliable_room,
                ];
            }

            //航线
            usleep(mt_rand(800, 2800));
            $tmp = $this->guzzleClient->request('GET', 'http://innfins.com/main/Index/getlinesforym?ship_id=' . $v['ship_id'] . '&ym=' . $month['value'], [
                'headers' => array_merge($this->baseHeader, [
                    'Referer' => $ship_url,
                    'X-Requested-With' => 'XMLHttpRequest',
                ]),
            ]);
            $ret = json_decode((string)$tmp->getBody(), true);
            if ($ret['state'] != 'success') {
                exit(date('Y-m-d H:i:s') . "|error lines for {$month['value']}\n");
            }
            if (empty($ret['data'])) {
                echo date('Y-m-d H:i:s') . "|empty lines for {$month['value']}\n";
                continue;
            }
            foreach ($ret['data'] as $item) {
                $photos = [];
                if (!empty($item['photos'])) {
                    foreach ($item['photos'] as $photo) {
                        $photos[] = $photo['url'];
                    }
                }
                if (!in_array($item['id'], $line_ids)) {
                    $lines[] = [
                        'id' => $item['id'],
                        'ship_id' => $item['ship_id'],
                        'name' => $item['name'],
                        'length' => $item['length'],
                        'dives' => $item['dives'],
                        'requires' => $item['requires'],
                        'photos' => $photos,
                        'departure' => $item['departure'],
                        'description' => $item['description'],
                        'return_port' => $item['return_port'],
                    ];
                    $line_ids[] = $item['id'];
                }
            }
        }

        $subCrawler = $crawler->filter('.explanation>.itm>span');
        $explanation = [];
        foreach ($subCrawler as $node) {
            $field = trim($node->textContent);
            $val = trim($node->nextSibling->nextSibling->textContent);
            $explanation[$field] = $val;
        }

        $subCrawler = $crawler->filter('.section-feature>.bd>.itm');
        $feature = [];
        foreach ($subCrawler as $node) {
            $grandcrawler = new \Symfony\Component\DomCrawler\Crawler($node, $ship_url);
            $field = $grandcrawler->filter('.name')->getNode(0)->textContent;
            $vals = [];
            $valNodes = $grandcrawler->filter('.cont>ul>li');
            foreach ($valNodes as $liNode) {
                $vals[] = $liNode->textContent;
            }
            $feature[$field] = $vals;
        }

        $photos = [];
        foreach ($v['photo'] as $photo) {
            $photos[] = $photo['url'];
        }
        $res = [
            '_id' => $v['ship_id'],
            'ship_url' => $ship_url,
            'ship_id' => $v['ship_id'],
            'name' => $v['name'],
            'ename' => $v['ename'],
            'cdestination' => $v['cdestination'],
            'create_y' => $v['create_y'],
            'photo' => $v['photo'],
            'evaluation_sentence' => $v['evaluation_sentence'],
            'imgs' => $imgs,
            'basic_info' => $basicInfo,
            'desc_brief' => $desc_brief,
            'desc_all' => $desc_all,
            'price' => $price,
            'shipment' => $shipment,
            'lines' => $lines,
            'shipmentroom' => $shipmentroom,
            'feature' => $feature,
            'explanation' => $explanation,
        ];
        echo date('Y-m-d H:i:s') . "|formatShip {$v['ship_id']} End\n";
        return $res;
    }
}
